package net.bwie.realtime.jtp.dwd.log.job;

import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.dwd.log.function.AdjustIsNewProcessFunction;
import net.bwie.realtime.jtp.dwd.log.function.AppLogSplitProcessFunction;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

/**
 * @author ASUS
 */
public class JtpAppLogEtljob {
    public static void main(String[] args) throws Exception {
        //1执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        //2数据源source
        DataStream<String> kafkaDataStream = KafkaUtil.consumerKafka(env, "topic-log");
//        kafkaDataStream.print("aaa");

        //3数据转换-transformation
        DataStream<String> pageStream = processLog(kafkaDataStream);

        //4数据输出sink
        KafkaUtil.producerKafka(pageStream, "dwd-traffic-page-log");

        //5出发执行execute
        env.execute("JtpAppLogEtljob");


    }

    private static DataStream<String> processLog(DataStream<String> stream) {
        //1数据清洗
        DataStream<String> jsonStream = appLogCleaned(stream);

        //2新老客户状态标记修复
        DataStream<String> etlStream = processIsNew(jsonStream);

        //3数据分流


        //返回数据流
        return splitStream(etlStream);
    }

    private static DataStream<String> splitStream(DataStream<String> stream) {
      final   OutputTag<String> errorTag = new OutputTag<String>("error-log") {
        };
        final  OutputTag<String> startTag = new OutputTag<String>("start-log") {
        };
        final  OutputTag<String> displayTag = new OutputTag<String>("display-log") {
        };
        final  OutputTag<String> actionTag = new OutputTag<String>("action-log") {
        };

        //分流
        SingleOutputStreamOperator<String> pageStream = stream.process(new AppLogSplitProcessFunction(errorTag, startTag, displayTag, actionTag));

        SideOutputDataStream<String> errorStream = pageStream.getSideOutput(errorTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-log");
        SideOutputDataStream<String> startStream = pageStream.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
        SideOutputDataStream<String> displayStream = pageStream.getSideOutput(displayTag);
        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-log");
        SideOutputDataStream<String> actionStream = pageStream.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");
        return pageStream;
    }

    private static DataStream<String> processIsNew(DataStream<String> stream) {
        KeyedStream<String, String> midStream = stream.keyBy(
                new KeySelector<String, String>() {
                    @Override
                    public String getKey(String value) throws Exception {
                        return JSON.parseObject(value).getJSONObject("common").getString("mid");
                    }
                }
        );
        return midStream.process(new AdjustIsNewProcessFunction());
    }

    private static DataStream<String> appLogCleaned(DataStream<String> stream) {
        OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log") {
        };



        SingleOutputStreamOperator<String> cleanedStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value,
                                       ProcessFunction<String, String>.Context ctx,
                                       Collector<String> out) throws Exception {
                try {
                    JSON.parseObject(value);
                    out.collect(value);
                } catch (Exception e) {
                    ctx.output(dirtyTag, value);
                }
            }
        });

        SideOutputDataStream<String> dirtyStream = cleanedStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-dirty-log");
        return cleanedStream;
    }
}
